1 package org.apache.lucene.search.join;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 import java.io.IOException;
21 import java.util.ArrayList;
22 import java.util.Arrays;
23 import java.util.Collections;
24 import java.util.List;
25 import java.util.Locale;
26
27 import org.apache.lucene.analysis.MockAnalyzer;
28 import org.apache.lucene.document.Document;
29 import org.apache.lucene.document.Field;
30 import org.apache.lucene.document.Field.Store;
31 import org.apache.lucene.document.IntField;
32 import org.apache.lucene.document.NumericDocValuesField;
33 import org.apache.lucene.document.SortedDocValuesField;
34 import org.apache.lucene.document.StoredField;
35 import org.apache.lucene.document.StringField;
36 import org.apache.lucene.index.DirectoryReader;
37 import org.apache.lucene.index.IndexReader;
38 import org.apache.lucene.index.IndexWriter;
39 import org.apache.lucene.index.IndexWriterConfig;
40 import org.apache.lucene.index.LeafReaderContext;
41 import org.apache.lucene.index.LogDocMergePolicy;
42 import org.apache.lucene.index.MultiFields;
43 import org.apache.lucene.index.NoMergePolicy;
44 import org.apache.lucene.index.PostingsEnum;
45 import org.apache.lucene.index.RandomIndexWriter;
46 import org.apache.lucene.index.ReaderUtil;
47 import org.apache.lucene.index.Term;
48 import org.apache.lucene.search.BooleanClause;
49 import org.apache.lucene.search.BooleanClause.Occur;
50 import org.apache.lucene.search.BooleanQuery;
51 import org.apache.lucene.search.BoostQuery;
52 import org.apache.lucene.search.DocIdSetIterator;
53 import org.apache.lucene.search.Explanation;
54 import org.apache.lucene.search.FieldDoc;
55 import org.apache.lucene.search.Filter;
56 import org.apache.lucene.search.FilteredQuery;
57 import org.apache.lucene.search.IndexSearcher;
58 import org.apache.lucene.search.MatchAllDocsQuery;
59 import org.apache.lucene.search.MatchNoDocsQuery;
60 import org.apache.lucene.search.MultiTermQuery;
61 import org.apache.lucene.search.NumericRangeQuery;
62 import org.apache.lucene.search.PrefixQuery;
63 import org.apache.lucene.search.Query;
64 import org.apache.lucene.search.QueryUtils;
65 import org.apache.lucene.search.QueryWrapperFilter;
66 import org.apache.lucene.search.RandomApproximationQuery;
67 import org.apache.lucene.search.ScoreDoc;
68 import org.apache.lucene.search.Sort;
69 import org.apache.lucene.search.SortField;
70 import org.apache.lucene.search.TermQuery;
71 import org.apache.lucene.search.TopDocs;
72 import org.apache.lucene.search.Weight;
73 import org.apache.lucene.search.grouping.GroupDocs;
74 import org.apache.lucene.search.grouping.TopGroups;
75 import org.apache.lucene.store.Directory;
76 import org.apache.lucene.util.BitSet;
77 import org.apache.lucene.util.Bits;
78 import org.apache.lucene.util.BytesRef;
79 import org.apache.lucene.util.BytesRefBuilder;
80 import org.apache.lucene.util.LuceneTestCase;
81 import org.apache.lucene.util.NumericUtils;
82 import org.apache.lucene.util.TestUtil;
83
84 public class TestBlockJoin extends LuceneTestCase {
85
86
87 private Document makeResume(String name, String country) {
88 Document resume = new Document();
89 resume.add(newStringField("docType", "resume", Field.Store.NO));
90 resume.add(newStringField("name", name, Field.Store.YES));
91 resume.add(newStringField("country", country, Field.Store.NO));
92 return resume;
93 }
94
95
96 private Document makeJob(String skill, int year) {
97 Document job = new Document();
98 job.add(newStringField("skill", skill, Field.Store.YES));
99 job.add(new IntField("year", year, Field.Store.NO));
100 job.add(new StoredField("year", year));
101 return job;
102 }
103
104
105 private Document makeQualification(String qualification, int year) {
106 Document job = new Document();
107 job.add(newStringField("qualification", qualification, Field.Store.YES));
108 job.add(new IntField("year", year, Field.Store.NO));
109 return job;
110 }
111
112 public void testEmptyChildFilter() throws Exception {
113 final Directory dir = newDirectory();
114 final IndexWriterConfig config = new IndexWriterConfig(new MockAnalyzer(random()));
115 config.setMergePolicy(NoMergePolicy.INSTANCE);
116
117 final IndexWriter w = new IndexWriter(dir, config);
118
119 final List<Document> docs = new ArrayList<>();
120
121 docs.add(makeJob("java", 2007));
122 docs.add(makeJob("python", 2010));
123 docs.add(makeResume("Lisa", "United Kingdom"));
124 w.addDocuments(docs);
125
126 docs.clear();
127 docs.add(makeJob("ruby", 2005));
128 docs.add(makeJob("java", 2006));
129 docs.add(makeResume("Frank", "United States"));
130 w.addDocuments(docs);
131 w.commit();
132
133 IndexReader r = DirectoryReader.open(w, random().nextBoolean());
134 w.close();
135 IndexSearcher s = new IndexSearcher(r);
136 BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume")));
137 CheckJoinIndex.check(r, parentsFilter);
138
139 BooleanQuery.Builder childQuery = new BooleanQuery.Builder();
140 childQuery.add(new BooleanClause(new TermQuery(new Term("skill", "java")), Occur.MUST));
141 childQuery.add(new BooleanClause(NumericRangeQuery.newIntRange("year", 2006, 2011, true, true), Occur.MUST));
142
143 ToParentBlockJoinQuery childJoinQuery = new ToParentBlockJoinQuery(childQuery.build(), parentsFilter, ScoreMode.Avg);
144
145 BooleanQuery.Builder fullQuery = new BooleanQuery.Builder();
146 fullQuery.add(new BooleanClause(childJoinQuery, Occur.MUST));
147 fullQuery.add(new BooleanClause(new MatchAllDocsQuery(), Occur.MUST));
148 ToParentBlockJoinCollector c = new ToParentBlockJoinCollector(Sort.RELEVANCE, 1, true, true);
149 s.search(fullQuery.build(), c);
150 TopGroups<Integer> results = c.getTopGroups(childJoinQuery, null, 0, 10, 0, true);
151 assertFalse(Float.isNaN(results.maxScore));
152 assertEquals(1, results.totalGroupedHitCount);
153 assertEquals(1, results.groups.length);
154 final GroupDocs<Integer> group = results.groups[0];
155 Document childDoc = s.doc(group.scoreDocs[0].doc);
156 assertEquals("java", childDoc.get("skill"));
157 assertNotNull(group.groupValue);
158 Document parentDoc = s.doc(group.groupValue);
159 assertEquals("Lisa", parentDoc.get("name"));
160
161 r.close();
162 dir.close();
163 }
164
165
166 public void testSimple() throws Exception {
167
168 final Directory dir = newDirectory();
169 final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
170
171 final List<Document> docs = new ArrayList<>();
172
173 docs.add(makeJob("java", 2007));
174 docs.add(makeJob("python", 2010));
175 docs.add(makeResume("Lisa", "United Kingdom"));
176 w.addDocuments(docs);
177
178 docs.clear();
179 docs.add(makeJob("ruby", 2005));
180 docs.add(makeJob("java", 2006));
181 docs.add(makeResume("Frank", "United States"));
182 w.addDocuments(docs);
183
184 IndexReader r = w.getReader();
185 w.close();
186 IndexSearcher s = newSearcher(r);
187
188
189 BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume")));
190 CheckJoinIndex.check(r, parentsFilter);
191
192
193 BooleanQuery.Builder childQuery = new BooleanQuery.Builder();
194 childQuery.add(new BooleanClause(new TermQuery(new Term("skill", "java")), Occur.MUST));
195 childQuery.add(new BooleanClause(NumericRangeQuery.newIntRange("year", 2006, 2011, true, true), Occur.MUST));
196
197
198 Query parentQuery = new TermQuery(new Term("country", "United Kingdom"));
199
200
201
202 ToParentBlockJoinQuery childJoinQuery = new ToParentBlockJoinQuery(childQuery.build(), parentsFilter, ScoreMode.Avg);
203
204
205 BooleanQuery.Builder fullQuery = new BooleanQuery.Builder();
206 fullQuery.add(new BooleanClause(parentQuery, Occur.MUST));
207 fullQuery.add(new BooleanClause(childJoinQuery, Occur.MUST));
208
209 ToParentBlockJoinCollector c = new ToParentBlockJoinCollector(Sort.RELEVANCE, 1, true, true);
210
211 s.search(fullQuery.build(), c);
212
213 TopGroups<Integer> results = c.getTopGroups(childJoinQuery, null, 0, 10, 0, true);
214 assertFalse(Float.isNaN(results.maxScore));
215
216
217 assertEquals(1, results.totalGroupedHitCount);
218 assertEquals(1, results.groups.length);
219
220 final GroupDocs<Integer> group = results.groups[0];
221 assertEquals(1, group.totalHits);
222 assertFalse(Float.isNaN(group.score));
223
224 Document childDoc = s.doc(group.scoreDocs[0].doc);
225
226 assertEquals("java", childDoc.get("skill"));
227 assertNotNull(group.groupValue);
228 Document parentDoc = s.doc(group.groupValue);
229 assertEquals("Lisa", parentDoc.get("name"));
230
231
232
233
234
235 ToChildBlockJoinQuery parentJoinQuery = new ToChildBlockJoinQuery(parentQuery, parentsFilter);
236 BooleanQuery.Builder fullChildQuery = new BooleanQuery.Builder();
237 fullChildQuery.add(new BooleanClause(parentJoinQuery, Occur.MUST));
238 fullChildQuery.add(new BooleanClause(childQuery.build(), Occur.MUST));
239
240
241 TopDocs hits = s.search(fullChildQuery.build(), 10);
242 assertEquals(1, hits.totalHits);
243 childDoc = s.doc(hits.scoreDocs[0].doc);
244
245 assertEquals("java", childDoc.get("skill"));
246 assertEquals(2007, childDoc.getField("year").numericValue());
247 assertEquals("Lisa", getParentDoc(r, parentsFilter, hits.scoreDocs[0].doc).get("name"));
248
249
250 assertEquals(0, s.search(new FilteredQuery(fullChildQuery.build(),
251 new QueryWrapperFilter(new TermQuery(new Term("skill", "foosball")))),
252 1).totalHits);
253
254 r.close();
255 dir.close();
256 }
257
258 public void testBugCausedByRewritingTwice() throws IOException {
259 final Directory dir = newDirectory();
260 final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
261
262 final List<Document> docs = new ArrayList<>();
263
264 for (int i=0;i<10;i++) {
265 docs.clear();
266 docs.add(makeJob("ruby", i));
267 docs.add(makeJob("java", 2007));
268 docs.add(makeResume("Frank", "United States"));
269 w.addDocuments(docs);
270 }
271
272 IndexReader r = w.getReader();
273 w.close();
274 IndexSearcher s = newSearcher(r);
275
276 MultiTermQuery qc = NumericRangeQuery.newIntRange("year", 2007, 2007, true, true);
277
278
279 qc.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_BOOLEAN_REWRITE);
280
281 BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume")));
282 CheckJoinIndex.check(r, parentsFilter);
283
284 int h1 = qc.hashCode();
285 Query qw1 = qc.rewrite(r);
286 int h2 = qw1.hashCode();
287 Query qw2 = qw1.rewrite(r);
288 int h3 = qw2.hashCode();
289
290 assertTrue(h1 != h2);
291 assertTrue(h2 != h3);
292 assertTrue(h3 != h1);
293
294 ToParentBlockJoinQuery qp = new ToParentBlockJoinQuery(qc, parentsFilter, ScoreMode.Max);
295 ToParentBlockJoinCollector c = new ToParentBlockJoinCollector(Sort.RELEVANCE, 10, true, true);
296
297 s.search(qp, c);
298 TopGroups<Integer> groups = c.getTopGroups(qp, Sort.INDEXORDER, 0, 10, 0, true);
299 for (GroupDocs<Integer> group : groups.groups) {
300 assertEquals(1, group.totalHits);
301 }
302
303 r.close();
304 dir.close();
305 }
306
307 protected Filter skill(String skill) {
308 return new QueryWrapperFilter(new TermQuery(new Term("skill", skill)));
309 }
310
311 public void testSimpleFilter() throws Exception {
312
313 final Directory dir = newDirectory();
314 final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
315
316 final List<Document> docs = new ArrayList<>();
317 docs.add(makeJob("java", 2007));
318 docs.add(makeJob("python", 2010));
319 Collections.shuffle(docs, random());
320 docs.add(makeResume("Lisa", "United Kingdom"));
321
322 final List<Document> docs2 = new ArrayList<>();
323 docs2.add(makeJob("ruby", 2005));
324 docs2.add(makeJob("java", 2006));
325 Collections.shuffle(docs2, random());
326 docs2.add(makeResume("Frank", "United States"));
327
328 addSkillless(w);
329 boolean turn = random().nextBoolean();
330 w.addDocuments(turn ? docs:docs2);
331
332 addSkillless(w);
333
334 w.addDocuments(!turn ? docs:docs2);
335
336 addSkillless(w);
337
338 IndexReader r = w.getReader();
339 w.close();
340 IndexSearcher s = newSearcher(r);
341
342
343 BitDocIdSetFilter parentsFilter = new BitDocIdSetCachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("docType", "resume"))));
344 CheckJoinIndex.check(r, parentsFilter);
345
346
347 BooleanQuery.Builder childQuery = new BooleanQuery.Builder();
348 childQuery.add(new BooleanClause(new TermQuery(new Term("skill", "java")), Occur.MUST));
349 childQuery.add(new BooleanClause(NumericRangeQuery.newIntRange("year", 2006, 2011, true, true), Occur.MUST));
350
351
352 Query parentQuery = new TermQuery(new Term("country", "United Kingdom"));
353
354
355
356 ToParentBlockJoinQuery childJoinQuery = new ToParentBlockJoinQuery(childQuery.build(), parentsFilter, ScoreMode.Avg);
357
358 assertEquals("no filter - both passed", 2, s.search(childJoinQuery, 10).totalHits);
359
360 BooleanQuery.Builder query = new BooleanQuery.Builder();
361 query.add(childJoinQuery, Occur.MUST);
362 query.add(new TermQuery(new Term("docType", "resume")), Occur.FILTER);
363 assertEquals("dummy filter passes everyone ", 2, s.search(query.build(), 10).totalHits);
364
365
366 assertEquals("noone live there", 0, s.search(new FilteredQuery(childJoinQuery, new BitDocIdSetCachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("country", "Oz"))))), 1).totalHits);
367
368
369 TopDocs ukOnly = s.search(new FilteredQuery(childJoinQuery, new QueryWrapperFilter(parentQuery)), 1);
370 assertEquals("has filter - single passed", 1, ukOnly.totalHits);
371 assertEquals( "Lisa", r.document(ukOnly.scoreDocs[0].doc).get("name"));
372
373
374 TopDocs usThen = s.search(new FilteredQuery(childJoinQuery , new QueryWrapperFilter(new TermQuery(new Term("country", "United States")))), 1);
375 assertEquals("has filter - single passed", 1, usThen.totalHits);
376 assertEquals("Frank", r.document(usThen.scoreDocs[0].doc).get("name"));
377
378
379 TermQuery us = new TermQuery(new Term("country", "United States"));
380 assertEquals("@ US we have java and ruby", 2,
381 s.search(new ToChildBlockJoinQuery(us,
382 parentsFilter), 10).totalHits );
383
384 assertEquals("java skills in US", 1, s.search(new FilteredQuery(new ToChildBlockJoinQuery(us, parentsFilter),
385 skill("java")), 10).totalHits );
386
387 BooleanQuery.Builder rubyPython = new BooleanQuery.Builder();
388 rubyPython.add(new TermQuery(new Term("skill", "ruby")), Occur.SHOULD);
389 rubyPython.add(new TermQuery(new Term("skill", "python")), Occur.SHOULD);
390 assertEquals("ruby skills in US", 1, s.search(new FilteredQuery(new ToChildBlockJoinQuery(us, parentsFilter),
391 new QueryWrapperFilter(rubyPython.build())), 10).totalHits );
392
393 r.close();
394 dir.close();
395 }
396
397 private void addSkillless(final RandomIndexWriter w) throws IOException {
398 if (random().nextBoolean()) {
399 w.addDocument(makeResume("Skillless", random().nextBoolean() ? "United Kingdom":"United States"));
400 }
401 }
402
403 private Document getParentDoc(IndexReader reader, BitSetProducer parents, int childDocID) throws IOException {
404 final List<LeafReaderContext> leaves = reader.leaves();
405 final int subIndex = ReaderUtil.subIndex(childDocID, leaves);
406 final LeafReaderContext leaf = leaves.get(subIndex);
407 final BitSet bits = parents.getBitSet(leaf);
408 return leaf.reader().document(bits.nextSetBit(childDocID - leaf.docBase));
409 }
410
411 public void testBoostBug() throws Exception {
412 final Directory dir = newDirectory();
413 final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
414 IndexReader r = w.getReader();
415 w.close();
416 IndexSearcher s = newSearcher(r);
417
418 ToParentBlockJoinQuery q = new ToParentBlockJoinQuery(new MatchNoDocsQuery(), new QueryBitSetProducer(new MatchAllDocsQuery()), ScoreMode.Avg);
419 QueryUtils.check(random(), q, s);
420 s.search(q, 10);
421 BooleanQuery.Builder bqB = new BooleanQuery.Builder();
422 bqB.add(q, BooleanClause.Occur.MUST);
423 BooleanQuery bq = bqB.build();
424 s.search(new BoostQuery(bq, 2f), 10);
425 r.close();
426 dir.close();
427 }
428
429 private String[][] getRandomFields(int maxUniqueValues) {
430
431 final String[][] fields = new String[TestUtil.nextInt(random(), 2, 4)][];
432 for(int fieldID=0;fieldID<fields.length;fieldID++) {
433 final int valueCount;
434 if (fieldID == 0) {
435 valueCount = 2;
436 } else {
437 valueCount = TestUtil.nextInt(random(), 1, maxUniqueValues);
438 }
439
440 final String[] values = fields[fieldID] = new String[valueCount];
441 for(int i=0;i<valueCount;i++) {
442 values[i] = TestUtil.randomRealisticUnicodeString(random());
443
444 }
445 }
446
447 return fields;
448 }
449
450 private Term randomParentTerm(String[] values) {
451 return new Term("parent0", values[random().nextInt(values.length)]);
452 }
453
454 private Term randomChildTerm(String[] values) {
455 return new Term("child0", values[random().nextInt(values.length)]);
456 }
457
458 private Sort getRandomSort(String prefix, int numFields) {
459 final List<SortField> sortFields = new ArrayList<>();
460
461
462
463 if (random().nextBoolean()) {
464 sortFields.add(new SortField(prefix + random().nextInt(numFields), SortField.Type.STRING, random().nextBoolean()));
465 } else if (random().nextBoolean()) {
466 sortFields.add(new SortField(prefix + random().nextInt(numFields), SortField.Type.STRING, random().nextBoolean()));
467 sortFields.add(new SortField(prefix + random().nextInt(numFields), SortField.Type.STRING, random().nextBoolean()));
468 }
469
470 sortFields.add(new SortField(prefix + "ID", SortField.Type.INT));
471 return new Sort(sortFields.toArray(new SortField[sortFields.size()]));
472 }
473
474 public void testRandom() throws Exception {
475
476
477
478
479 final Directory dir = newDirectory();
480 final Directory joinDir = newDirectory();
481
482 final int numParentDocs = TestUtil.nextInt(random(), 100 * RANDOM_MULTIPLIER, 300 * RANDOM_MULTIPLIER);
483
484
485
486 final String[][] parentFields = getRandomFields(numParentDocs/2);
487
488 final String[][] childFields = getRandomFields(numParentDocs);
489
490 final boolean doDeletes = random().nextBoolean();
491 final List<Integer> toDelete = new ArrayList<>();
492
493
494 final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
495 final RandomIndexWriter joinW = new RandomIndexWriter(random(), joinDir);
496 for(int parentDocID=0;parentDocID<numParentDocs;parentDocID++) {
497 Document parentDoc = new Document();
498 Document parentJoinDoc = new Document();
499 Field id = new IntField("parentID", parentDocID, Field.Store.YES);
500 parentDoc.add(id);
501 parentJoinDoc.add(id);
502 parentJoinDoc.add(newStringField("isParent", "x", Field.Store.NO));
503 id = new NumericDocValuesField("parentID", parentDocID);
504 parentDoc.add(id);
505 parentJoinDoc.add(id);
506 parentJoinDoc.add(newStringField("isParent", "x", Field.Store.NO));
507 for(int field=0;field<parentFields.length;field++) {
508 if (random().nextDouble() < 0.9) {
509 String s = parentFields[field][random().nextInt(parentFields[field].length)];
510 Field f = newStringField("parent" + field, s, Field.Store.NO);
511 parentDoc.add(f);
512 parentJoinDoc.add(f);
513
514 f = new SortedDocValuesField("parent" + field, new BytesRef(s));
515 parentDoc.add(f);
516 parentJoinDoc.add(f);
517 }
518 }
519
520 if (doDeletes) {
521 parentDoc.add(new IntField("blockID", parentDocID, Field.Store.NO));
522 parentJoinDoc.add(new IntField("blockID", parentDocID, Field.Store.NO));
523 }
524
525 final List<Document> joinDocs = new ArrayList<>();
526
527 if (VERBOSE) {
528 StringBuilder sb = new StringBuilder();
529 sb.append("parentID=").append(parentDoc.get("parentID"));
530 for(int fieldID=0;fieldID<parentFields.length;fieldID++) {
531 String s = parentDoc.get("parent" + fieldID);
532 if (s != null) {
533 sb.append(" parent" + fieldID + "=" + s);
534 }
535 }
536 System.out.println(" " + sb.toString());
537 }
538
539 final int numChildDocs = TestUtil.nextInt(random(), 1, 20);
540 for(int childDocID=0;childDocID<numChildDocs;childDocID++) {
541
542 Document childDoc = TestUtil.cloneDocument(parentDoc);
543 Document joinChildDoc = new Document();
544 joinDocs.add(joinChildDoc);
545
546 Field childID = new IntField("childID", childDocID, Field.Store.YES);
547 childDoc.add(childID);
548 joinChildDoc.add(childID);
549 childID = new NumericDocValuesField("childID", childDocID);
550 childDoc.add(childID);
551 joinChildDoc.add(childID);
552
553 for(int childFieldID=0;childFieldID<childFields.length;childFieldID++) {
554 if (random().nextDouble() < 0.9) {
555 String s = childFields[childFieldID][random().nextInt(childFields[childFieldID].length)];
556 Field f = newStringField("child" + childFieldID, s, Field.Store.NO);
557 childDoc.add(f);
558 joinChildDoc.add(f);
559
560 f = new SortedDocValuesField("child" + childFieldID, new BytesRef(s));
561 childDoc.add(f);
562 joinChildDoc.add(f);
563 }
564 }
565
566 if (VERBOSE) {
567 StringBuilder sb = new StringBuilder();
568 sb.append("childID=").append(joinChildDoc.get("childID"));
569 for(int fieldID=0;fieldID<childFields.length;fieldID++) {
570 String s = joinChildDoc.get("child" + fieldID);
571 if (s != null) {
572 sb.append(" child" + fieldID + "=" + s);
573 }
574 }
575 System.out.println(" " + sb.toString());
576 }
577
578 if (doDeletes) {
579 joinChildDoc.add(new IntField("blockID", parentDocID, Field.Store.NO));
580 }
581
582 w.addDocument(childDoc);
583 }
584
585
586 joinDocs.add(parentJoinDoc);
587 joinW.addDocuments(joinDocs);
588
589 if (doDeletes && random().nextInt(30) == 7) {
590 toDelete.add(parentDocID);
591 }
592 }
593
594 BytesRefBuilder term = new BytesRefBuilder();
595 for(int deleteID : toDelete) {
596 if (VERBOSE) {
597 System.out.println("DELETE parentID=" + deleteID);
598 }
599 NumericUtils.intToPrefixCodedBytes(deleteID, 0, term);
600 w.deleteDocuments(new Term("blockID", term.toBytesRef()));
601 joinW.deleteDocuments(new Term("blockID", term.toBytesRef()));
602 }
603
604 final IndexReader r = w.getReader();
605 w.close();
606 final IndexReader joinR = joinW.getReader();
607 joinW.close();
608
609 if (VERBOSE) {
610 System.out.println("TEST: reader=" + r);
611 System.out.println("TEST: joinReader=" + joinR);
612
613 Bits liveDocs = MultiFields.getLiveDocs(joinR);
614 for(int docIDX=0;docIDX<joinR.maxDoc();docIDX++) {
615 System.out.println(" docID=" + docIDX + " doc=" + joinR.document(docIDX) + " deleted?=" + (liveDocs != null && liveDocs.get(docIDX) == false));
616 }
617 PostingsEnum parents = MultiFields.getTermDocsEnum(joinR, "isParent", new BytesRef("x"));
618 System.out.println("parent docIDs:");
619 while (parents.nextDoc() != PostingsEnum.NO_MORE_DOCS) {
620 System.out.println(" " + parents.docID());
621 }
622 }
623
624 final IndexSearcher s = newSearcher(r);
625
626 final IndexSearcher joinS = new IndexSearcher(joinR);
627
628 final BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("isParent", "x")));
629 CheckJoinIndex.check(joinS.getIndexReader(), parentsFilter);
630
631 final int iters = 200*RANDOM_MULTIPLIER;
632
633 for(int iter=0;iter<iters;iter++) {
634 if (VERBOSE) {
635 System.out.println("TEST: iter=" + (1+iter) + " of " + iters);
636 }
637
638 final Query childQuery;
639 if (random().nextInt(3) == 2) {
640 final int childFieldID = random().nextInt(childFields.length);
641 childQuery = new TermQuery(new Term("child" + childFieldID,
642 childFields[childFieldID][random().nextInt(childFields[childFieldID].length)]));
643 } else if (random().nextInt(3) == 2) {
644 BooleanQuery.Builder bq = new BooleanQuery.Builder();
645 final int numClauses = TestUtil.nextInt(random(), 2, 4);
646 boolean didMust = false;
647 for(int clauseIDX=0;clauseIDX<numClauses;clauseIDX++) {
648 Query clause;
649 BooleanClause.Occur occur;
650 if (!didMust && random().nextBoolean()) {
651 occur = random().nextBoolean() ? BooleanClause.Occur.MUST : BooleanClause.Occur.MUST_NOT;
652 clause = new TermQuery(randomChildTerm(childFields[0]));
653 didMust = true;
654 } else {
655 occur = BooleanClause.Occur.SHOULD;
656 final int childFieldID = TestUtil.nextInt(random(), 1, childFields.length - 1);
657 clause = new TermQuery(new Term("child" + childFieldID,
658 childFields[childFieldID][random().nextInt(childFields[childFieldID].length)]));
659 }
660 bq.add(clause, occur);
661 }
662 childQuery = bq.build();
663 } else {
664 BooleanQuery.Builder bq = new BooleanQuery.Builder();
665
666 bq.add(new TermQuery(randomChildTerm(childFields[0])),
667 BooleanClause.Occur.MUST);
668 final int childFieldID = TestUtil.nextInt(random(), 1, childFields.length - 1);
669 bq.add(new TermQuery(new Term("child" + childFieldID, childFields[childFieldID][random().nextInt(childFields[childFieldID].length)])),
670 random().nextBoolean() ? BooleanClause.Occur.MUST : BooleanClause.Occur.MUST_NOT);
671 childQuery = bq.build();
672 }
673
674
675 final ScoreMode agg = ScoreMode.values()[random().nextInt(ScoreMode.values().length)];
676 final ToParentBlockJoinQuery childJoinQuery = new ToParentBlockJoinQuery(childQuery, parentsFilter, agg);
677
678
679 final Query parentJoinQuery;
680
681
682
683
684 final Query parentQuery;
685
686 if (random().nextBoolean()) {
687 parentQuery = childQuery;
688 parentJoinQuery = childJoinQuery;
689 } else {
690
691 final BooleanQuery.Builder bq = new BooleanQuery.Builder();
692 final Term parentTerm = randomParentTerm(parentFields[0]);
693 if (random().nextBoolean()) {
694 bq.add(childJoinQuery, BooleanClause.Occur.MUST);
695 bq.add(new TermQuery(parentTerm),
696 BooleanClause.Occur.MUST);
697 } else {
698 bq.add(new TermQuery(parentTerm),
699 BooleanClause.Occur.MUST);
700 bq.add(childJoinQuery, BooleanClause.Occur.MUST);
701 }
702
703 final BooleanQuery.Builder bq2 = new BooleanQuery.Builder();
704 if (random().nextBoolean()) {
705 bq2.add(childQuery, BooleanClause.Occur.MUST);
706 bq2.add(new TermQuery(parentTerm),
707 BooleanClause.Occur.MUST);
708 } else {
709 bq2.add(new TermQuery(parentTerm),
710 BooleanClause.Occur.MUST);
711 bq2.add(childQuery, BooleanClause.Occur.MUST);
712 }
713 parentJoinQuery = bq.build();
714 parentQuery = bq2.build();
715 }
716
717 final Sort parentSort = getRandomSort("parent", parentFields.length);
718 final Sort childSort = getRandomSort("child", childFields.length);
719
720 if (VERBOSE) {
721 System.out.println("\nTEST: query=" + parentQuery + " joinQuery=" + parentJoinQuery + " parentSort=" + parentSort + " childSort=" + childSort);
722 }
723
724
725 final List<SortField> sortFields = new ArrayList<>(Arrays.asList(parentSort.getSort()));
726 sortFields.addAll(Arrays.asList(childSort.getSort()));
727 final Sort parentAndChildSort = new Sort(sortFields.toArray(new SortField[sortFields.size()]));
728
729 final TopDocs results = s.search(parentQuery, r.numDocs(),
730 parentAndChildSort);
731
732 if (VERBOSE) {
733 System.out.println("\nTEST: normal index gets " + results.totalHits + " hits; sort=" + parentAndChildSort);
734 final ScoreDoc[] hits = results.scoreDocs;
735 for(int hitIDX=0;hitIDX<hits.length;hitIDX++) {
736 final Document doc = s.doc(hits[hitIDX].doc);
737
738 System.out.println(" parentID=" + doc.get("parentID") + " childID=" + doc.get("childID") + " (docID=" + hits[hitIDX].doc + ")");
739 FieldDoc fd = (FieldDoc) hits[hitIDX];
740 if (fd.fields != null) {
741 System.out.print(" " + fd.fields.length + " sort values: ");
742 for(Object o : fd.fields) {
743 if (o instanceof BytesRef) {
744 System.out.print(((BytesRef) o).utf8ToString() + " ");
745 } else {
746 System.out.print(o + " ");
747 }
748 }
749 System.out.println();
750 }
751 }
752 }
753
754 final boolean trackScores;
755 final boolean trackMaxScore;
756 if (agg == ScoreMode.None) {
757 trackScores = false;
758 trackMaxScore = false;
759 } else {
760 trackScores = random().nextBoolean();
761 trackMaxScore = random().nextBoolean();
762 }
763 final ToParentBlockJoinCollector c = new ToParentBlockJoinCollector(parentSort, 10, trackScores, trackMaxScore);
764
765 joinS.search(parentJoinQuery, c);
766
767 final int hitsPerGroup = TestUtil.nextInt(random(), 1, 20);
768
769 final TopGroups<Integer> joinResults = c.getTopGroups(childJoinQuery, childSort, 0, hitsPerGroup, 0, true);
770
771 if (VERBOSE) {
772 System.out.println("\nTEST: block join index gets " + (joinResults == null ? 0 : joinResults.groups.length) + " groups; hitsPerGroup=" + hitsPerGroup);
773 if (joinResults != null) {
774 final GroupDocs<Integer>[] groups = joinResults.groups;
775 for(int groupIDX=0;groupIDX<groups.length;groupIDX++) {
776 final GroupDocs<Integer> group = groups[groupIDX];
777 if (group.groupSortValues != null) {
778 System.out.print(" ");
779 for(Object o : group.groupSortValues) {
780 if (o instanceof BytesRef) {
781 System.out.print(((BytesRef) o).utf8ToString() + " ");
782 } else {
783 System.out.print(o + " ");
784 }
785 }
786 System.out.println();
787 }
788
789 assertNotNull(group.groupValue);
790 final Document parentDoc = joinS.doc(group.groupValue);
791 System.out.println(" group parentID=" + parentDoc.get("parentID") + " (docID=" + group.groupValue + ")");
792 for(int hitIDX=0;hitIDX<group.scoreDocs.length;hitIDX++) {
793 final Document doc = joinS.doc(group.scoreDocs[hitIDX].doc);
794
795 System.out.println(" childID=" + doc.get("childID") + " child0=" + doc.get("child0") + " (docID=" + group.scoreDocs[hitIDX].doc + ")");
796 }
797 }
798 }
799 }
800
801 if (results.totalHits == 0) {
802 assertNull(joinResults);
803 } else {
804 compareHits(r, joinR, results, joinResults);
805 TopDocs b = joinS.search(childJoinQuery, 10);
806 for (ScoreDoc hit : b.scoreDocs) {
807 Explanation explanation = joinS.explain(childJoinQuery, hit.doc);
808 Document document = joinS.doc(hit.doc - 1);
809 int childId = Integer.parseInt(document.get("childID"));
810
811 assertTrue(explanation.isMatch());
812 assertEquals(hit.score, explanation.getValue(), 0.0f);
813 assertEquals(String.format(Locale.ROOT, "Score based on child doc range from %d to %d", hit.doc - 1 - childId, hit.doc - 1), explanation.getDescription());
814 }
815 }
816
817
818
819
820
821 final Query parentQuery2;
822 if (random().nextInt(3) == 2) {
823 final int fieldID = random().nextInt(parentFields.length);
824 parentQuery2 = new TermQuery(new Term("parent" + fieldID,
825 parentFields[fieldID][random().nextInt(parentFields[fieldID].length)]));
826 } else if (random().nextInt(3) == 2) {
827 BooleanQuery.Builder bq = new BooleanQuery.Builder();
828 final int numClauses = TestUtil.nextInt(random(), 2, 4);
829 boolean didMust = false;
830 for(int clauseIDX=0;clauseIDX<numClauses;clauseIDX++) {
831 Query clause;
832 BooleanClause.Occur occur;
833 if (!didMust && random().nextBoolean()) {
834 occur = random().nextBoolean() ? BooleanClause.Occur.MUST : BooleanClause.Occur.MUST_NOT;
835 clause = new TermQuery(randomParentTerm(parentFields[0]));
836 didMust = true;
837 } else {
838 occur = BooleanClause.Occur.SHOULD;
839 final int fieldID = TestUtil.nextInt(random(), 1, parentFields.length - 1);
840 clause = new TermQuery(new Term("parent" + fieldID,
841 parentFields[fieldID][random().nextInt(parentFields[fieldID].length)]));
842 }
843 bq.add(clause, occur);
844 }
845 parentQuery2 = bq.build();
846 } else {
847 BooleanQuery.Builder bq = new BooleanQuery.Builder();
848
849 bq.add(new TermQuery(randomParentTerm(parentFields[0])),
850 BooleanClause.Occur.MUST);
851 final int fieldID = TestUtil.nextInt(random(), 1, parentFields.length - 1);
852 bq.add(new TermQuery(new Term("parent" + fieldID, parentFields[fieldID][random().nextInt(parentFields[fieldID].length)])),
853 random().nextBoolean() ? BooleanClause.Occur.MUST : BooleanClause.Occur.MUST_NOT);
854 parentQuery2 = bq.build();
855 }
856
857 if (VERBOSE) {
858 System.out.println("\nTEST: top down: parentQuery2=" + parentQuery2);
859 }
860
861
862 final ToChildBlockJoinQuery parentJoinQuery2 = new ToChildBlockJoinQuery(parentQuery2, parentsFilter);
863
864
865 Query childJoinQuery2;
866
867
868
869
870 Query childQuery2;
871
872 if (random().nextBoolean()) {
873 childQuery2 = parentQuery2;
874 childJoinQuery2 = parentJoinQuery2;
875 } else {
876 final Term childTerm = randomChildTerm(childFields[0]);
877 final Filter f = new QueryWrapperFilter(new TermQuery(childTerm));
878 if (random().nextBoolean()) {
879 childJoinQuery2 = parentJoinQuery2;
880 childJoinQuery2 = new FilteredQuery(childJoinQuery2, random().nextBoolean()
881 ? new BitDocIdSetCachingWrapperFilter(f): f);
882 } else {
883
884 final BooleanQuery.Builder bq = new BooleanQuery.Builder();
885 if (random().nextBoolean()) {
886 bq.add(parentJoinQuery2, BooleanClause.Occur.MUST);
887 bq.add(new TermQuery(childTerm),
888 BooleanClause.Occur.MUST);
889 } else {
890 bq.add(new TermQuery(childTerm),
891 BooleanClause.Occur.MUST);
892 bq.add(parentJoinQuery2, BooleanClause.Occur.MUST);
893 }
894 childJoinQuery2 = bq.build();
895 }
896
897 if (random().nextBoolean()) {
898 childQuery2 = parentQuery2;
899 childQuery2 = new FilteredQuery(childQuery2, random().nextBoolean()
900 ? new BitDocIdSetCachingWrapperFilter(f): f);
901 } else {
902 final BooleanQuery.Builder bq2 = new BooleanQuery.Builder();
903 if (random().nextBoolean()) {
904 bq2.add(parentQuery2, BooleanClause.Occur.MUST);
905 bq2.add(new TermQuery(childTerm),
906 BooleanClause.Occur.MUST);
907 } else {
908 bq2.add(new TermQuery(childTerm),
909 BooleanClause.Occur.MUST);
910 bq2.add(parentQuery2, BooleanClause.Occur.MUST);
911 }
912 childQuery2 = bq2.build();
913 }
914 }
915
916 final Sort childSort2 = getRandomSort("child", childFields.length);
917
918
919 if (VERBOSE) {
920 System.out.println("TEST: run top down query=" + childQuery2 + " sort=" + childSort2);
921 }
922 final TopDocs results2 = s.search(childQuery2, r.numDocs(),
923 childSort2);
924 if (VERBOSE) {
925 System.out.println(" " + results2.totalHits + " totalHits:");
926 for(ScoreDoc sd : results2.scoreDocs) {
927 final Document doc = s.doc(sd.doc);
928 System.out.println(" childID=" + doc.get("childID") + " parentID=" + doc.get("parentID") + " docID=" + sd.doc);
929 }
930 }
931
932
933 if (VERBOSE) {
934 System.out.println("TEST: run top down join query=" + childJoinQuery2 + " sort=" + childSort2);
935 }
936 TopDocs joinResults2 = joinS.search(childJoinQuery2, joinR.numDocs(), childSort2);
937 if (VERBOSE) {
938 System.out.println(" " + joinResults2.totalHits + " totalHits:");
939 for(ScoreDoc sd : joinResults2.scoreDocs) {
940 final Document doc = joinS.doc(sd.doc);
941 final Document parentDoc = getParentDoc(joinR, parentsFilter, sd.doc);
942 System.out.println(" childID=" + doc.get("childID") + " parentID=" + parentDoc.get("parentID") + " docID=" + sd.doc);
943 }
944 }
945
946 compareChildHits(r, joinR, results2, joinResults2);
947 }
948
949 r.close();
950 joinR.close();
951 dir.close();
952 joinDir.close();
953 }
954
955 private void compareChildHits(IndexReader r, IndexReader joinR, TopDocs results, TopDocs joinResults) throws Exception {
956 assertEquals(results.totalHits, joinResults.totalHits);
957 assertEquals(results.scoreDocs.length, joinResults.scoreDocs.length);
958 for(int hitCount=0;hitCount<results.scoreDocs.length;hitCount++) {
959 ScoreDoc hit = results.scoreDocs[hitCount];
960 ScoreDoc joinHit = joinResults.scoreDocs[hitCount];
961 Document doc1 = r.document(hit.doc);
962 Document doc2 = joinR.document(joinHit.doc);
963 assertEquals("hit " + hitCount + " differs",
964 doc1.get("childID"), doc2.get("childID"));
965
966
967
968 assertTrue(hit instanceof FieldDoc);
969 assertTrue(joinHit instanceof FieldDoc);
970
971 FieldDoc hit0 = (FieldDoc) hit;
972 FieldDoc joinHit0 = (FieldDoc) joinHit;
973 assertArrayEquals(hit0.fields, joinHit0.fields);
974 }
975 }
976
977 private void compareHits(IndexReader r, IndexReader joinR, TopDocs results, TopGroups<Integer> joinResults) throws Exception {
978
979 int resultUpto = 0;
980 int joinGroupUpto = 0;
981
982 final ScoreDoc[] hits = results.scoreDocs;
983 final GroupDocs<Integer>[] groupDocs = joinResults.groups;
984
985 while(joinGroupUpto < groupDocs.length) {
986 final GroupDocs<Integer> group = groupDocs[joinGroupUpto++];
987 final ScoreDoc[] groupHits = group.scoreDocs;
988 assertNotNull(group.groupValue);
989 final Document parentDoc = joinR.document(group.groupValue);
990 final String parentID = parentDoc.get("parentID");
991
992 assertNotNull(parentID);
993 assertTrue(groupHits.length > 0);
994 for(int hitIDX=0;hitIDX<groupHits.length;hitIDX++) {
995 final Document nonJoinHit = r.document(hits[resultUpto++].doc);
996 final Document joinHit = joinR.document(groupHits[hitIDX].doc);
997 assertEquals(parentID,
998 nonJoinHit.get("parentID"));
999 assertEquals(joinHit.get("childID"),
1000 nonJoinHit.get("childID"));
1001 }
1002
1003 if (joinGroupUpto < groupDocs.length) {
1004
1005
1006 while(true) {
1007 assertTrue(resultUpto < hits.length);
1008 if (!parentID.equals(r.document(hits[resultUpto].doc).get("parentID"))) {
1009 break;
1010 }
1011 resultUpto++;
1012 }
1013 }
1014 }
1015 }
1016
1017 public void testMultiChildTypes() throws Exception {
1018
1019 final Directory dir = newDirectory();
1020 final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
1021
1022 final List<Document> docs = new ArrayList<>();
1023
1024 docs.add(makeJob("java", 2007));
1025 docs.add(makeJob("python", 2010));
1026 docs.add(makeQualification("maths", 1999));
1027 docs.add(makeResume("Lisa", "United Kingdom"));
1028 w.addDocuments(docs);
1029
1030 IndexReader r = w.getReader();
1031 w.close();
1032 IndexSearcher s = newSearcher(r);
1033
1034
1035 BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume")));
1036 CheckJoinIndex.check(s.getIndexReader(), parentsFilter);
1037
1038
1039 BooleanQuery.Builder childJobQuery = new BooleanQuery.Builder();
1040 childJobQuery.add(new BooleanClause(new TermQuery(new Term("skill", "java")), Occur.MUST));
1041 childJobQuery.add(new BooleanClause(NumericRangeQuery.newIntRange("year", 2006, 2011, true, true), Occur.MUST));
1042
1043 BooleanQuery.Builder childQualificationQuery = new BooleanQuery.Builder();
1044 childQualificationQuery.add(new BooleanClause(new TermQuery(new Term("qualification", "maths")), Occur.MUST));
1045 childQualificationQuery.add(new BooleanClause(NumericRangeQuery.newIntRange("year", 1980, 2000, true, true), Occur.MUST));
1046
1047
1048
1049 Query parentQuery = new TermQuery(new Term("country", "United Kingdom"));
1050
1051
1052
1053 ToParentBlockJoinQuery childJobJoinQuery = new ToParentBlockJoinQuery(childJobQuery.build(), parentsFilter, ScoreMode.Avg);
1054 ToParentBlockJoinQuery childQualificationJoinQuery = new ToParentBlockJoinQuery(childQualificationQuery.build(), parentsFilter, ScoreMode.Avg);
1055
1056
1057 BooleanQuery.Builder fullQuery = new BooleanQuery.Builder();
1058 fullQuery.add(new BooleanClause(parentQuery, Occur.MUST));
1059 fullQuery.add(new BooleanClause(childJobJoinQuery, Occur.MUST));
1060 fullQuery.add(new BooleanClause(childQualificationJoinQuery, Occur.MUST));
1061
1062
1063
1064 ToParentBlockJoinCollector c = new ToParentBlockJoinCollector(Sort.RELEVANCE, 10, true, false);
1065
1066 s.search(fullQuery.build(), c);
1067
1068
1069 TopGroups<Integer> jobResults = c.getTopGroups(childJobJoinQuery, null, 0, 10, 0, true);
1070
1071
1072 assertEquals(1, jobResults.totalGroupedHitCount);
1073 assertEquals(1, jobResults.groups.length);
1074
1075 final GroupDocs<Integer> group = jobResults.groups[0];
1076 assertEquals(1, group.totalHits);
1077
1078 Document childJobDoc = s.doc(group.scoreDocs[0].doc);
1079
1080 assertEquals("java", childJobDoc.get("skill"));
1081 assertNotNull(group.groupValue);
1082 Document parentDoc = s.doc(group.groupValue);
1083 assertEquals("Lisa", parentDoc.get("name"));
1084
1085
1086 TopGroups<Integer> qualificationResults = c.getTopGroups(childQualificationJoinQuery, null, 0, 10, 0, true);
1087
1088 assertEquals(1, qualificationResults.totalGroupedHitCount);
1089 assertEquals(1, qualificationResults.groups.length);
1090
1091 final GroupDocs<Integer> qGroup = qualificationResults.groups[0];
1092 assertEquals(1, qGroup.totalHits);
1093
1094 Document childQualificationDoc = s.doc(qGroup.scoreDocs[0].doc);
1095 assertEquals("maths", childQualificationDoc.get("qualification"));
1096 assertNotNull(qGroup.groupValue);
1097 parentDoc = s.doc(qGroup.groupValue);
1098 assertEquals("Lisa", parentDoc.get("name"));
1099
1100 r.close();
1101 dir.close();
1102 }
1103
1104 public void testAdvanceSingleParentSingleChild() throws Exception {
1105 Directory dir = newDirectory();
1106 RandomIndexWriter w = new RandomIndexWriter(random(), dir);
1107 Document childDoc = new Document();
1108 childDoc.add(newStringField("child", "1", Field.Store.NO));
1109 Document parentDoc = new Document();
1110 parentDoc.add(newStringField("parent", "1", Field.Store.NO));
1111 w.addDocuments(Arrays.asList(childDoc, parentDoc));
1112 IndexReader r = w.getReader();
1113 w.close();
1114 IndexSearcher s = newSearcher(r);
1115 Query tq = new TermQuery(new Term("child", "1"));
1116 BitSetProducer parentFilter = new QueryBitSetProducer(
1117 new TermQuery(new Term("parent", "1")));
1118 CheckJoinIndex.check(s.getIndexReader(), parentFilter);
1119
1120 ToParentBlockJoinQuery q = new ToParentBlockJoinQuery(tq, parentFilter, ScoreMode.Avg);
1121 Weight weight = s.createNormalizedWeight(q, true);
1122 DocIdSetIterator disi = weight.scorer(s.getIndexReader().leaves().get(0));
1123 assertEquals(1, disi.advance(1));
1124 r.close();
1125 dir.close();
1126 }
1127
1128 public void testAdvanceSingleParentNoChild() throws Exception {
1129 Directory dir = newDirectory();
1130 RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(new LogDocMergePolicy()));
1131 Document parentDoc = new Document();
1132 parentDoc.add(newStringField("parent", "1", Field.Store.NO));
1133 parentDoc.add(newStringField("isparent", "yes", Field.Store.NO));
1134 w.addDocuments(Arrays.asList(parentDoc));
1135
1136
1137 parentDoc = new Document();
1138 parentDoc.add(newStringField("parent", "2", Field.Store.NO));
1139 parentDoc.add(newStringField("isparent", "yes", Field.Store.NO));
1140 Document childDoc = new Document();
1141 childDoc.add(newStringField("child", "2", Field.Store.NO));
1142 w.addDocuments(Arrays.asList(childDoc, parentDoc));
1143
1144
1145 w.forceMerge(1);
1146 IndexReader r = w.getReader();
1147 w.close();
1148 IndexSearcher s = newSearcher(r);
1149 Query tq = new TermQuery(new Term("child", "2"));
1150 BitSetProducer parentFilter = new QueryBitSetProducer(
1151 new TermQuery(new Term("isparent", "yes")));
1152 CheckJoinIndex.check(s.getIndexReader(), parentFilter);
1153
1154 ToParentBlockJoinQuery q = new ToParentBlockJoinQuery(tq, parentFilter, ScoreMode.Avg);
1155 Weight weight = s.createNormalizedWeight(q, true);
1156 DocIdSetIterator disi = weight.scorer(s.getIndexReader().leaves().get(0));
1157 assertEquals(2, disi.advance(0));
1158 r.close();
1159 dir.close();
1160 }
1161
1162 public void testGetTopGroups() throws Exception {
1163
1164 final Directory dir = newDirectory();
1165 final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
1166
1167 final List<Document> docs = new ArrayList<>();
1168 docs.add(makeJob("ruby", 2005));
1169 docs.add(makeJob("java", 2006));
1170 docs.add(makeJob("java", 2010));
1171 docs.add(makeJob("java", 2012));
1172 Collections.shuffle(docs, random());
1173 docs.add(makeResume("Frank", "United States"));
1174
1175 addSkillless(w);
1176 w.addDocuments(docs);
1177 addSkillless(w);
1178
1179 IndexReader r = w.getReader();
1180 w.close();
1181 IndexSearcher s = new IndexSearcher(r);
1182
1183
1184 BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume")));
1185 CheckJoinIndex.check(s.getIndexReader(), parentsFilter);
1186
1187
1188 BooleanQuery.Builder childQuery = new BooleanQuery.Builder();
1189 childQuery.add(new BooleanClause(new TermQuery(new Term("skill", "java")), Occur.MUST));
1190 childQuery.add(new BooleanClause(NumericRangeQuery.newIntRange("year", 2006, 2011, true, true), Occur.MUST));
1191
1192
1193
1194 ToParentBlockJoinQuery childJoinQuery = new ToParentBlockJoinQuery(childQuery.build(), parentsFilter, ScoreMode.Avg);
1195
1196 ToParentBlockJoinCollector c = new ToParentBlockJoinCollector(Sort.RELEVANCE, 2, true, true);
1197 s.search(childJoinQuery, c);
1198
1199
1200 @SuppressWarnings({"unchecked","rawtypes"})
1201 TopGroups<Integer>[] getTopGroupsResults = new TopGroups[2];
1202 getTopGroupsResults[0] = c.getTopGroups(childJoinQuery, null, 0, 10, 0, true);
1203 getTopGroupsResults[1] = c.getTopGroupsWithAllChildDocs(childJoinQuery, null, 0, 0, true);
1204
1205 for (TopGroups<Integer> results : getTopGroupsResults) {
1206 assertFalse(Float.isNaN(results.maxScore));
1207 assertEquals(2, results.totalGroupedHitCount);
1208 assertEquals(1, results.groups.length);
1209
1210 final GroupDocs<Integer> group = results.groups[0];
1211 assertEquals(2, group.totalHits);
1212 assertFalse(Float.isNaN(group.score));
1213 assertNotNull(group.groupValue);
1214 Document parentDoc = s.doc(group.groupValue);
1215 assertEquals("Frank", parentDoc.get("name"));
1216
1217 assertEquals(2, group.scoreDocs.length);
1218
1219 for (ScoreDoc scoreDoc : group.scoreDocs) {
1220 Document childDoc = s.doc(scoreDoc.doc);
1221 assertEquals("java", childDoc.get("skill"));
1222 int year = Integer.parseInt(childDoc.get("year"));
1223 assertTrue(year >= 2006 && year <= 2011);
1224 }
1225 }
1226
1227
1228 TopGroups<Integer> boundedResults = c.getTopGroups(childJoinQuery, null, 0, 1, 0, true);
1229 assertFalse(Float.isNaN(boundedResults.maxScore));
1230 assertEquals(2, boundedResults.totalGroupedHitCount);
1231 assertEquals(1, boundedResults.groups.length);
1232
1233 final GroupDocs<Integer> group = boundedResults.groups[0];
1234 assertEquals(2, group.totalHits);
1235 assertFalse(Float.isNaN(group.score));
1236 assertNotNull(group.groupValue);
1237 Document parentDoc = s.doc(group.groupValue);
1238 assertEquals("Frank", parentDoc.get("name"));
1239
1240 assertEquals(1, group.scoreDocs.length);
1241
1242 for (ScoreDoc scoreDoc : group.scoreDocs) {
1243 Document childDoc = s.doc(scoreDoc.doc);
1244 assertEquals("java", childDoc.get("skill"));
1245 int year = Integer.parseInt(childDoc.get("year"));
1246 assertTrue(year >= 2006 && year <= 2011);
1247 }
1248
1249 r.close();
1250 dir.close();
1251 }
1252
1253
1254 public void testSometimesParentOnlyMatches() throws Exception {
1255 Directory d = newDirectory();
1256 RandomIndexWriter w = new RandomIndexWriter(random(), d);
1257 Document parent = new Document();
1258 parent.add(new StoredField("parentID", "0"));
1259 parent.add(new SortedDocValuesField("parentID", new BytesRef("0")));
1260 parent.add(newTextField("parentText", "text", Field.Store.NO));
1261 parent.add(newStringField("isParent", "yes", Field.Store.NO));
1262
1263 List<Document> docs = new ArrayList<>();
1264
1265 Document child = new Document();
1266 docs.add(child);
1267 child.add(new StoredField("childID", "0"));
1268 child.add(newTextField("childText", "text", Field.Store.NO));
1269
1270
1271 docs.add(parent);
1272 w.addDocuments(docs);
1273
1274 docs.clear();
1275
1276 parent = new Document();
1277 parent.add(newTextField("parentText", "text", Field.Store.NO));
1278 parent.add(newStringField("isParent", "yes", Field.Store.NO));
1279 parent.add(new StoredField("parentID", "1"));
1280 parent.add(new SortedDocValuesField("parentID", new BytesRef("1")));
1281
1282
1283 docs.add(parent);
1284 w.addDocuments(docs);
1285
1286 IndexReader r = w.getReader();
1287 w.close();
1288
1289 IndexSearcher searcher = new ToParentBlockJoinIndexSearcher(r);
1290 Query childQuery = new TermQuery(new Term("childText", "text"));
1291 BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("isParent", "yes")));
1292 CheckJoinIndex.check(r, parentsFilter);
1293 ToParentBlockJoinQuery childJoinQuery = new ToParentBlockJoinQuery(childQuery, parentsFilter, ScoreMode.Avg);
1294 BooleanQuery.Builder parentQuery = new BooleanQuery.Builder();
1295 parentQuery.add(childJoinQuery, Occur.SHOULD);
1296 parentQuery.add(new TermQuery(new Term("parentText", "text")), Occur.SHOULD);
1297
1298 ToParentBlockJoinCollector c = new ToParentBlockJoinCollector(new Sort(new SortField("parentID", SortField.Type.STRING)),
1299 10, true, true);
1300 searcher.search(parentQuery.build(), c);
1301 TopGroups<Integer> groups = c.getTopGroups(childJoinQuery, null, 0, 10, 0, false);
1302
1303
1304 assertEquals(2, groups.totalGroupCount.intValue());
1305
1306
1307 assertEquals(1, groups.totalGroupedHitCount);
1308
1309 GroupDocs<Integer> group = groups.groups[0];
1310 Document doc = r.document(group.groupValue.intValue());
1311 assertEquals("0", doc.get("parentID"));
1312
1313 group = groups.groups[1];
1314 doc = r.document(group.groupValue.intValue());
1315 assertEquals("1", doc.get("parentID"));
1316
1317 r.close();
1318 d.close();
1319 }
1320
1321
1322 public void testChildQueryNeverMatches() throws Exception {
1323 Directory d = newDirectory();
1324 RandomIndexWriter w = new RandomIndexWriter(random(), d);
1325 Document parent = new Document();
1326 parent.add(new StoredField("parentID", "0"));
1327 parent.add(new SortedDocValuesField("parentID", new BytesRef("0")));
1328 parent.add(newTextField("parentText", "text", Field.Store.NO));
1329 parent.add(newStringField("isParent", "yes", Field.Store.NO));
1330
1331 List<Document> docs = new ArrayList<>();
1332
1333 Document child = new Document();
1334 docs.add(child);
1335 child.add(new StoredField("childID", "0"));
1336 child.add(newTextField("childText", "text", Field.Store.NO));
1337
1338
1339 docs.add(parent);
1340 w.addDocuments(docs);
1341
1342 docs.clear();
1343
1344 parent = new Document();
1345 parent.add(newTextField("parentText", "text", Field.Store.NO));
1346 parent.add(newStringField("isParent", "yes", Field.Store.NO));
1347 parent.add(new StoredField("parentID", "1"));
1348 parent.add(new SortedDocValuesField("parentID", new BytesRef("1")));
1349
1350
1351
1352 docs.add(parent);
1353 w.addDocuments(docs);
1354
1355 IndexReader r = w.getReader();
1356 w.close();
1357
1358 IndexSearcher searcher = new ToParentBlockJoinIndexSearcher(r);
1359
1360
1361 Query childQuery = new TermQuery(new Term("childText", "bogus"));
1362 BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("isParent", "yes")));
1363 CheckJoinIndex.check(r, parentsFilter);
1364 ToParentBlockJoinQuery childJoinQuery = new ToParentBlockJoinQuery(childQuery, parentsFilter, ScoreMode.Avg);
1365 BooleanQuery.Builder parentQuery = new BooleanQuery.Builder();
1366 parentQuery.add(childJoinQuery, Occur.SHOULD);
1367 parentQuery.add(new TermQuery(new Term("parentText", "text")), Occur.SHOULD);
1368
1369 ToParentBlockJoinCollector c = new ToParentBlockJoinCollector(new Sort(new SortField("parentID", SortField.Type.STRING)),
1370 10, true, true);
1371 searcher.search(parentQuery.build(), c);
1372 TopGroups<Integer> groups = c.getTopGroups(childJoinQuery, null, 0, 10, 0, false);
1373
1374
1375 assertEquals(2, groups.totalGroupCount.intValue());
1376
1377
1378 assertEquals(0, groups.totalGroupedHitCount);
1379
1380 GroupDocs<Integer> group = groups.groups[0];
1381 Document doc = r.document(group.groupValue.intValue());
1382 assertEquals("0", doc.get("parentID"));
1383
1384 group = groups.groups[1];
1385 doc = r.document(group.groupValue.intValue());
1386 assertEquals("1", doc.get("parentID"));
1387
1388 r.close();
1389 d.close();
1390 }
1391
1392
1393 public void testChildQueryMatchesParent() throws Exception {
1394 Directory d = newDirectory();
1395 RandomIndexWriter w = new RandomIndexWriter(random(), d);
1396 Document parent = new Document();
1397 parent.add(new StoredField("parentID", "0"));
1398 parent.add(newTextField("parentText", "text", Field.Store.NO));
1399 parent.add(newStringField("isParent", "yes", Field.Store.NO));
1400
1401 List<Document> docs = new ArrayList<>();
1402
1403 Document child = new Document();
1404 docs.add(child);
1405 child.add(new StoredField("childID", "0"));
1406 child.add(newTextField("childText", "text", Field.Store.NO));
1407
1408
1409 docs.add(parent);
1410 w.addDocuments(docs);
1411
1412 docs.clear();
1413
1414 parent = new Document();
1415 parent.add(newTextField("parentText", "text", Field.Store.NO));
1416 parent.add(newStringField("isParent", "yes", Field.Store.NO));
1417 parent.add(new StoredField("parentID", "1"));
1418
1419
1420 docs.add(parent);
1421 w.addDocuments(docs);
1422
1423 IndexReader r = w.getReader();
1424 w.close();
1425
1426
1427 Query childQuery = new TermQuery(new Term("parentText", "text"));
1428 BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("isParent", "yes")));
1429 CheckJoinIndex.check(r, parentsFilter);
1430 ToParentBlockJoinQuery childJoinQuery = new ToParentBlockJoinQuery(childQuery, parentsFilter, ScoreMode.Avg);
1431 BooleanQuery.Builder parentQuery = new BooleanQuery.Builder();
1432 parentQuery.add(childJoinQuery, Occur.SHOULD);
1433 parentQuery.add(new TermQuery(new Term("parentText", "text")), Occur.SHOULD);
1434
1435 ToParentBlockJoinCollector c = new ToParentBlockJoinCollector(new Sort(new SortField("parentID", SortField.Type.STRING)),
1436 10, true, true);
1437
1438 try {
1439 newSearcher(r).search(parentQuery.build(), c);
1440 fail("should have hit exception");
1441 } catch (IllegalStateException ise) {
1442
1443 }
1444
1445 r.close();
1446 d.close();
1447 }
1448
1449 public void testAdvanceSingleDeletedParentNoChild() throws Exception {
1450
1451 final Directory dir = newDirectory();
1452 final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
1453
1454
1455 Document parentDoc = new Document();
1456 parentDoc.add(newStringField("parent", "1", Field.Store.NO));
1457 parentDoc.add(newStringField("isparent", "yes", Field.Store.NO));
1458 Document childDoc = new Document();
1459 childDoc.add(newStringField("child", "1", Field.Store.NO));
1460 w.addDocuments(Arrays.asList(childDoc, parentDoc));
1461
1462 parentDoc = new Document();
1463 parentDoc.add(newStringField("parent", "2", Field.Store.NO));
1464 parentDoc.add(newStringField("isparent", "yes", Field.Store.NO));
1465 w.addDocuments(Arrays.asList(parentDoc));
1466
1467 w.deleteDocuments(new Term("parent", "2"));
1468
1469 parentDoc = new Document();
1470 parentDoc.add(newStringField("parent", "2", Field.Store.NO));
1471 parentDoc.add(newStringField("isparent", "yes", Field.Store.NO));
1472 childDoc = new Document();
1473 childDoc.add(newStringField("child", "2", Field.Store.NO));
1474 w.addDocuments(Arrays.asList(childDoc, parentDoc));
1475
1476 IndexReader r = w.getReader();
1477 w.close();
1478 IndexSearcher s = newSearcher(r);
1479
1480
1481 BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("isparent", "yes")));
1482 CheckJoinIndex.check(r, parentsFilter);
1483
1484 Query parentQuery = new TermQuery(new Term("parent", "2"));
1485
1486 ToChildBlockJoinQuery parentJoinQuery = new ToChildBlockJoinQuery(parentQuery, parentsFilter);
1487 TopDocs topdocs = s.search(parentJoinQuery, 3);
1488 assertEquals(1, topdocs.totalHits);
1489
1490 r.close();
1491 dir.close();
1492 }
1493
1494 public void testIntersectionWithRandomApproximation() throws IOException {
1495 final Directory dir = newDirectory();
1496 final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
1497
1498 final int numBlocks = atLeast(100);
1499 for (int i = 0; i < numBlocks; ++i) {
1500 List<Document> docs = new ArrayList<>();
1501 final int numChildren = random().nextInt(3);
1502 for (int j = 0; j < numChildren; ++j) {
1503 Document child = new Document();
1504 child.add(new StringField("foo_child", random().nextBoolean() ? "bar" : "baz", Store.NO));
1505 docs.add(child);
1506 }
1507 Document parent = new Document();
1508 parent.add(new StringField("parent", "true", Store.NO));
1509 parent.add(new StringField("foo_parent", random().nextBoolean() ? "bar" : "baz", Store.NO));
1510 docs.add(parent);
1511 w.addDocuments(docs);
1512 }
1513 final IndexReader reader = w.getReader();
1514 final IndexSearcher searcher = newSearcher(reader);
1515 searcher.setQueryCache(null);
1516
1517 final BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("parent", "true")));
1518 final Query toChild = new ToChildBlockJoinQuery(new TermQuery(new Term("foo_parent", "bar")), parentsFilter);
1519 final Query childQuery = new TermQuery(new Term("foo_child", "baz"));
1520
1521 BooleanQuery.Builder bq1 = new BooleanQuery.Builder();
1522 bq1.add(toChild, Occur.MUST);
1523 bq1.add(childQuery, Occur.MUST);
1524 BooleanQuery.Builder bq2 = new BooleanQuery.Builder();
1525 bq2.add(toChild, Occur.MUST);
1526 bq2.add(new RandomApproximationQuery(childQuery, random()), Occur.MUST);
1527
1528 assertEquals(searcher.count(bq1.build()), searcher.count(bq2.build()));
1529
1530 searcher.getIndexReader().close();
1531 w.close();
1532 dir.close();
1533 }
1534
1535
1536
1537 public void testParentScoringBug() throws Exception {
1538 final Directory dir = newDirectory();
1539 final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
1540
1541 final List<Document> docs = new ArrayList<>();
1542 docs.add(makeJob("java", 2007));
1543 docs.add(makeJob("python", 2010));
1544 docs.add(makeResume("Lisa", "United Kingdom"));
1545 w.addDocuments(docs);
1546
1547 docs.clear();
1548 docs.add(makeJob("java", 2006));
1549 docs.add(makeJob("ruby", 2005));
1550 docs.add(makeResume("Frank", "United States"));
1551 w.addDocuments(docs);
1552 w.deleteDocuments(new Term("skill", "java"));
1553
1554 IndexReader r = w.getReader();
1555 w.close();
1556 IndexSearcher s = newSearcher(r);
1557
1558
1559 BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume")));
1560 Query parentQuery = new PrefixQuery(new Term("country", "United"));
1561
1562 ToChildBlockJoinQuery toChildQuery = new ToChildBlockJoinQuery(parentQuery, parentsFilter);
1563
1564 TopDocs hits = s.search(toChildQuery, 10);
1565 assertEquals(hits.scoreDocs.length, 2);
1566 for (int i = 0; i < hits.scoreDocs.length; i++) {
1567 if (hits.scoreDocs[i].score == 0.0)
1568 fail("Failed to calculate score for hit #"+i);
1569 }
1570
1571 r.close();
1572 dir.close();
1573 }
1574
1575 public void testToChildBlockJoinQueryExplain() throws Exception {
1576 final Directory dir = newDirectory();
1577 final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
1578
1579 final List<Document> docs = new ArrayList<>();
1580 docs.add(makeJob("java", 2007));
1581 docs.add(makeJob("python", 2010));
1582 docs.add(makeResume("Lisa", "United Kingdom"));
1583 w.addDocuments(docs);
1584
1585 docs.clear();
1586 docs.add(makeJob("java", 2006));
1587 docs.add(makeJob("ruby", 2005));
1588 docs.add(makeResume("Frank", "United States"));
1589 w.addDocuments(docs);
1590 w.deleteDocuments(new Term("skill", "java"));
1591
1592 IndexReader r = w.getReader();
1593 w.close();
1594 IndexSearcher s = newSearcher(r);
1595
1596
1597 BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume")));
1598 Query parentQuery = new PrefixQuery(new Term("country", "United"));
1599
1600 ToChildBlockJoinQuery toChildQuery = new ToChildBlockJoinQuery(parentQuery, parentsFilter);
1601
1602 TopDocs hits = s.search(toChildQuery, 10);
1603 assertEquals(hits.scoreDocs.length, 2);
1604 for (int i = 0; i < hits.scoreDocs.length; i++) {
1605 assertEquals(hits.scoreDocs[i].score, s.explain(toChildQuery, hits.scoreDocs[i].doc).getValue(), 0.01);
1606 }
1607
1608 r.close();
1609 dir.close();
1610 }
1611
1612 public void testToChildInitialAdvanceParentButNoKids() throws Exception {
1613
1614 final Directory dir = newDirectory();
1615 final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
1616
1617
1618 w.addDocument(makeResume("first", "nokids"));
1619 w.addDocuments(Arrays.asList(makeJob("job", 42), makeResume("second", "haskid")));
1620
1621
1622 w.forceMerge(1);
1623
1624 final IndexReader r = w.getReader();
1625 final IndexSearcher s = newSearcher(r);
1626 w.close();
1627
1628 BitSetProducer parentFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume")));
1629 Query parentQuery = new TermQuery(new Term("docType", "resume"));
1630
1631 ToChildBlockJoinQuery parentJoinQuery = new ToChildBlockJoinQuery(parentQuery, parentFilter);
1632
1633 Weight weight = s.createNormalizedWeight(parentJoinQuery, random().nextBoolean());
1634 DocIdSetIterator advancingScorer = weight.scorer(s.getIndexReader().leaves().get(0));
1635 DocIdSetIterator nextDocScorer = weight.scorer(s.getIndexReader().leaves().get(0));
1636
1637 final int firstKid = nextDocScorer.nextDoc();
1638 assertTrue("firstKid not found", DocIdSetIterator.NO_MORE_DOCS != firstKid);
1639 assertEquals(firstKid, advancingScorer.advance(0));
1640
1641 r.close();
1642 dir.close();
1643 }
1644
1645 public void testMultiChildQueriesOfDiffParentLevels() throws Exception {
1646
1647 final Directory dir = newDirectory();
1648 final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
1649
1650
1651 final int numResumes = atLeast(100);
1652 for (int r = 0; r < numResumes; r++) {
1653 final List<Document> docs = new ArrayList<>();
1654
1655 final int rv = TestUtil.nextInt(random(), 1, 10);
1656 final int numJobs = atLeast(10);
1657 for (int j = 0; j < numJobs; j++) {
1658 final int jv = TestUtil.nextInt(random(), -10, -1);
1659
1660 final int numQualifications = atLeast(10);
1661 for (int q = 0; q < numQualifications; q++) {
1662 docs.add(makeQualification("q" + q + "_rv" + rv + "_jv" + jv, q));
1663 }
1664 docs.add(makeJob("j" + j, jv));
1665 }
1666 docs.add(makeResume("r" + r, "rv"+rv));
1667 w.addDocuments(docs);
1668 }
1669
1670 final IndexReader r = w.getReader();
1671 final IndexSearcher s = newSearcher(r);
1672 w.close();
1673
1674 BitSetProducer resumeFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume")));
1675
1676 BitSetProducer jobFilter = new QueryBitSetProducer(new PrefixQuery(new Term("skill", "")));
1677
1678
1679 final int numQueryIters = atLeast(1);
1680 for (int i = 0; i < numQueryIters; i++) {
1681 final int qjv = TestUtil.nextInt(random(), -10, -1);
1682 final int qrv = TestUtil.nextInt(random(), 1, 10);
1683
1684 Query resumeQuery = new ToChildBlockJoinQuery(new TermQuery(new Term("country","rv" + qrv)),
1685 resumeFilter);
1686
1687 Query jobQuery = new ToChildBlockJoinQuery(NumericRangeQuery.newIntRange("year", qjv, qjv, true, true),
1688 jobFilter);
1689
1690 BooleanQuery.Builder fullQuery = new BooleanQuery.Builder();
1691 fullQuery.add(new BooleanClause(jobQuery, Occur.MUST));
1692 fullQuery.add(new BooleanClause(resumeQuery, Occur.MUST));
1693
1694 TopDocs hits = s.search(fullQuery.build(), 100);
1695
1696 for (ScoreDoc sd : hits.scoreDocs) {
1697
1698 String q = r.document(sd.doc).get("qualification");
1699 assertNotNull(sd.doc + " has no qualification", q);
1700 assertTrue(q + " MUST contain jv" + qjv, q.contains("jv"+qjv));
1701 assertTrue(q + " MUST contain rv" + qrv, q.contains("rv"+qrv));
1702 }
1703 }
1704
1705 r.close();
1706 dir.close();
1707 }
1708
1709
1710 }